﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Diagnostics;
using log4net;

namespace StockFinder.IndexUrlGenerator
{
    public class Engine
    {
        private static readonly ILog _Log = LogManager.GetLogger(typeof(Engine));

        public void Run()
        {
            const string TOP_LEVEL_REGEX_PATTERN = "<td nowrap><a href=([\\w\\s/-]+\\.html)>([\\w-]+)</a></td>";
            const string SECOND_LEVEL_REGEX_PATTERN = "<td nowrap[=\"\\w\\s]*><a href=([\"\\w\\s/-]+\\.html[\"]?)>([\\w\\s-]+)[<>\\(\\)\\w/]*</a></td>";
            const string BASE_URL = "http://biz.yahoo.com";


            /* webrequest the base index*/

            _Log.InfoFormat("Making base request...");
            
            //make the request
            HttpWebRequest webRequest = HttpWebRequest.Create("http://biz.yahoo.com/i/") as HttpWebRequest;

            //try get the response
            HttpWebResponse webResponse = webRequest.GetResponse() as HttpWebResponse;

            string responsePageSource = null;

            //create the reader to extract the page source
            using (StreamReader streamReader = new StreamReader(webResponse.GetResponseStream()))
            {
                _Log.InfoFormat("Reading page source...");

                //get the page source
                responsePageSource = streamReader.ReadToEnd();
            }

            webResponse = null;
            webRequest = null;

            /* extract all top levels */

            //tidy up page source
            responsePageSource = responsePageSource.Replace("\n", " ");

            //regex the indexs
            MatchCollection matches = Regex.Matches(responsePageSource, TOP_LEVEL_REGEX_PATTERN, RegexOptions.Multiline);

            _Log.InfoFormat("Found {0} matches", matches.Count);

            List<TopLevelIndex> topLevelIndexes = new List<TopLevelIndex>();
            
            foreach (Match match in matches)
            {
                if (match.Groups.Count == 3)
                {
                    TopLevelIndex topLevelIndex = new TopLevelIndex();
                    topLevelIndex.Url = string.Format("{0}{1}", BASE_URL, match.Groups[1].Value);
                    topLevelIndex.Title = match.Groups[2].Value.Trim();

                    _Log.InfoFormat("Top Level: {0} {1}", topLevelIndex.Url, topLevelIndex.Title);

                    topLevelIndexes.Add(topLevelIndex);

                    _Log.InfoFormat("Added");
                }
            }

            Debug.WriteLine(topLevelIndexes.Count);

            /* webrequest each top level */

            foreach (TopLevelIndex topLevelIndex in topLevelIndexes)
            {
                _Log.InfoFormat("Making second level request for '{0}'", topLevelIndex.Url);

                webRequest = HttpWebRequest.Create(topLevelIndex.Url) as HttpWebRequest;

                webResponse = webRequest.GetResponse() as HttpWebResponse;

                using (StreamReader streamReader = new StreamReader(webResponse.GetResponseStream()))
                {
                    _Log.InfoFormat("Reading page source...");

                    //get the page source
                    responsePageSource = streamReader.ReadToEnd();
                } 
                
                /* extract all second levels on page */

                responsePageSource = responsePageSource.Replace("\n", " ");

                matches = Regex.Matches(responsePageSource, SECOND_LEVEL_REGEX_PATTERN, RegexOptions.Multiline);

                _Log.InfoFormat("Found {0} matches", matches.Count);

                topLevelIndex.SecondLevels = new List<SecondLevelIndex>();

                foreach (Match match in matches)
                {
                    if (match.Groups.Count == 3)
                    {
                        string url = string.Format("{0}{1}", BASE_URL, match.Groups[1].Value);
                        string title = match.Groups[2].Value.Trim();

                        _Log.InfoFormat("SecondLevel: {0} {1}", url, title);

                        //have we accidently found a top level?
                        if (topLevelIndexes.Exists(delegate(TopLevelIndex t) { return title == t.Title; }))
                        {
                            _Log.InfoFormat("Ignoring because already a top level");
                        }
                        else
                        {                            
                            //no so create a secondlevel component
                            SecondLevelIndex secondLevelIndex = new SecondLevelIndex();
                            secondLevelIndex.Url = url;
                            secondLevelIndex.Title = title.Trim();

                            topLevelIndex.SecondLevels.Add(secondLevelIndex);

                            _Log.InfoFormat("Added");
                        }
                    }
                }

            }

            _Log.InfoFormat("Completed...");
        
        }
    }
}
